#-*- coding:utf-8 -*- 
import codecs 
from whoosh.index import create_in  
from whoosh.fields import *  
from chinesetokenizer import ChineseAnalyzer
#from whoosh.analysis import RegexAnalyzer  
#analyzer = RegexAnalyzer(ur"([\u4e00-\u9fa5])|(\w+(\.?\w+)*)")

from whoosh.index import open_dir

from whoosh.qparser import QueryParser



ix = open_dir("IndexDir/titleIndex");
parser = QueryParser('content',ix.schema);

filter = open('unmatchLog', 'w');
with open('../operLog', 'r') as f:
    with ix.searcher() as searcher:
        count = 0;
        total  = 0;
        uncount = 0;
        for line in f:
            li = line.strip().split('\t');
            if len(li) != 3:
                continue;
            queryString = li[2];
            appID = li[0];
            docs = searcher.documents(appid = appID);
            if docs == None:
                continue;
            for doc in docs:
                content = doc['content'].strip().lower();
                appid = doc['appid'];
                queryString = queryString.decode('utf-8').lower();
                if queryString in content or content in queryString:
                    count += 1;
                else:
                    filter.write('%s\t%s\t%s\n' %(content.encode('utf-8'), queryString.encode('utf-8'),appid.encode('utf-8')));
                    uncount += 1;

            total += 1;

filter.close();
print 'total:%d, count:%d, uncount:%d' % (total, count, uncount);
